# -*- coding: utf-8 -*-

import requests
import time
import pymongo
from lxml import etree
from requests.exceptions import RequestException



def get_one_page(page):
    '''获取单页源码'''
    try:
        url = "http://hz.ziroom.com/z/nl/z2.html?p=" + str(page)
        headers = {
            'Referer':'http://hz.ziroom.com/',
            'Upgrade-Insecure-Requests':'1',
            'User-Agent':'Mozilla/5.0(WindowsNT6.3;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/68.0.3440.106Safari/537.36'
         }
        res = requests.get(url,headers=headers)
        if res.status_code == 200:
            return res.text
        return None
    except RequestException:
        return None


def parse_one_page(sourcehtml):
    '''解析单页源码'''
    contentTree = etree.HTML(sourcehtml)   #解析源代码
    results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
    for result in results[1:]:
        title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
        location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
        area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1)  # 使用join方法将列表中的内容以" "字符连接
        # nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() # 这里需要加判断, 改写为下句
        nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()")) > 0 else ""
        data = {
        "title": title,
        "location": location,
        "area": area,
        "nearby": nearby
         }
        save_to_mongodb(data)
    #yield {"pages":pages}


def get_pages():
    """得到总页数"""
    page = 1
    html = get_one_page(page)
    contentTree = etree.HTML(html)
    pages = int(contentTree.xpath('//div[@class="pages"]/span[2]/text()')[0].strip("共页"))
    return pages


def save_to_mongodb(result):
    """存储到MongoDB中"""
    # 创建数据库连接对象, 即连接到本地
    client = pymongo.MongoClient(host="localhost")
    # 指定数据库,这里指定ziroom
    db = client.iroomz
    # 指定表的名称, 这里指定roominfo
    db_table = db.roominfo
    try:
        #存储到数据库
        if db_table.insert(result):
            print("---存储到数据库成功---", result)
    except Exception:
        print("---存储到数据库失败---", result)


def main():
    pages = get_pages()
    print(pages)
    for page in range(1,pages+1):
     html = get_one_page(page)
     parse_one_page(html)


if __name__ == '__main__':
    main()
    time.sleep(1)